# -*- coding: utf-8 -*-
import urllib
import random
from bs4 import BeautifulSoup

import header

headers= header.my_headers



def get_content(url,headers):
    random_header=random.choice(headers)
    req=urllib.request.Request(url)
    req.add_header("User-Agent", random_header)
#    req.add_header("Host", "www.dytt8.net")
#    req.add_header("Referer", "http://www.dytt8.net/")
    req.add_header("GET", url)
    try:
        response = urllib.request.urlopen(req,timeout=30)
        content = response.read()
    except:
        response = urllib.request.urlopen(req,timeout=30)
        content = response.read()
    return content.decode('gb2312','ignore')


def select_url(url,headers):
    content = get_content(url,headers)
    soup = BeautifulSoup(content,"lxml")
    aa=soup.select('#li-change-color > li  > div > h3')
    return aa

############################  main   ###############################

f = open('temp.html','a+',encoding='gb2312')

for i  in  range(240,347):
    url="http://www.downcc.com/soft/list_181_%d.html" % (i+1)


    print(url)

    titles=select_url(url,headers)


    for temp in titles:
        title=temp.select('a')[0]
        print( title.get_text())
        print( title['href'])
        str2=title.get_text()
        str1=title['href']
        f.write( '<a href="http://www.downcc.com%s"> %s </a> <br /> \n' % (str1,str2)  )
f.close()
print( 'over'  )

